##Construct a submission
args<-commandArgs(TRUE)
filename<-paste("Test",args[1],".csv",sep="")


df_test<-read.csv(filename, stringsAsFactors = F, header=F, col.names=c("Id","Title","Body"))

tag.freq <- table(unlist(strsplit(df_sample$Tags," ")))                    ##Get tag names and their frequency on the training data
tag.freq <- tag.freq[rev(order(tag.freq))]
tag.names<-names(tag.freq)
tag.map<-read.csv('tag_map.csv', header=T, stringsAsFactors=F)           ##tag_map.csv has regexps selected for top 300 tags
more.tags<-setdiff(tag.names[1:num.tags], tag.map$tag)
more.regexp<-gsub("-",".", more.tags,fixed=T)
more.regexp<-gsub("#","\\#", more.regexp,fixed=T)
more.regexp<-gsub("+","\\+", more.regexp,fixed=T)
tag.map<-rbind(tag.map, data.frame(tag=more.tags, regexp=more.regexp))
v.text <- tolower(paste(df_sample$Title, df_sample$Body, sep=" "))
v.title <- tolower(df_sample$Title)
tag.names <- tag.map$tag

test.predictions<-rep("", length(df_test$Title))
perf.info<-read.csv("perf_info.csv", stringsAsFactors=F, header=T)

for(i in 1:nrow(perf.info))
{
	print(i)
	flush.console()
	x<-perf.info$tag.map.regexp[i]
	ind<-grep(x, v.text, perl=T)
	test.predictions[ind]<-paste(test.predictions[ind],perf.info$tag[i])
}

test.predictions<-substring(test.predictions, 2)

num.to.keep<-function(p)
{
  a<-vector("list",length(p))
  a<-lapply(1:length(a), function(i) a[[i]]<-c(0,1))
  outcomes<-expand.grid(a)
  prob.outcome<-sapply(1:nrow(outcomes),function(i)prod(p[unlist(outcomes[i,]>0)])*prod(1-p[unlist(outcomes[i,]==0)]))
  Ef<-rep(NA, length(p))
  for(j in 1:length(p))
  {
    a.try<-rep(0, length(p))
    a.try[1:j]<-1
    tp<-as.matrix(outcomes)%*%a.try
    prec<-tp/j
    rec<-tp/apply(outcomes, 1, sum)
    f<-2*prec*rec/(prec+rec)
    f[is.na(f)]<-0
    Ef[j]<-sum(f*prob.outcome)
    #cat(j, Ef, "\n")
  }
  if(max(Ef)<.1) 0
  else which.max(Ef)
}

##Now predictions must be ordered by specificity and only the most powerful ones retained
for(i in 1:length(test.predictions)){
  print(i)
  if(test.predictions[i]=="") next
  v_pred<-unlist(strsplit(test.predictions[i]," "))
  v_spec<-sapply(1:length(v_pred),function(j) perf.info$spec[perf.info$tag==v_pred[j]])
  v_pred<-v_pred[rev(order(v_spec))]
  v_spec<-v_spec[rev(order(v_spec))]
  if (length(v_pred)>5)
  {
    v_pred<-v_pred[1:5]
    v_spec<-v_spec[1:5]
  }
  n<-num.to.keep(v_spec)
  cat(length(v_pred), " predictions, keeping ", n, "\n")
  if (n==0)
    test.predictions[i]<-""
  else
    test.predictions[i]<-paste(v_pred[1:n],collapse=" ")
}

test.predictions[test.predictions==""]<-"c# java javascript c++"


outfile<-paste("submission",args[1],".csv",sep="")
df_submission<-data.frame(Id=df_test$Id, Tags=test.predictions)
write.csv(df_submission, outfile, row.names=FALSE)
